% Suite of functions to push autoTelnetDriver-acquired moorings data into 
% the moorings database 
%
% Author: David Goldsmith, Wash. State Dept. of Ecology, dgol461@ecy.wa.gov
% Original release date: < 9/15/2011

function success = PushData_dev(target, autoRun, EN_recip, DB, subfuncAccess)
    global oEN hParseInput hPushData IM
    IM = false;
    if ~nargin % Called w/out args = interactive mode
        IM = true;
        AR = false;
        [target, pth, ~] = uigetfile('', 'Please choose a file to push.');
    else
        pth = '';
    end
    if nargin < 5 || subfuncAccess
        hParseInput = @parseInput; hPushData = @pushData;
    else
        hParseInput = ''; hPushData = '';
    end
    if nargin < 4
        DB = 'Mooring_COTS_Dev';
    end
    if nargin < 3
        EN_recip = {'dgol461@ecy.wa.gov'};%, 'bsac461@ecy.wa.gov'};
    end
    if nargin < 2
        AR = false;
    else
        AR = autoRun;
    end
    oEN = error_notification(EN_recip);
    if isnumeric(target)
        success = ['Error: PushData called without an input file name, and '...
                   'user cancelled out of choosing one.  '];
    else
        success = pushData(parseInput(fullfile(pth,target)), DB, AR);
    end
    if exist('pth', 'var')
        msgbox(success, 'PushData Exit Status', 'modal');
    end
    clear('all');
end

function out = parseInput(target)
    global IM
    if isdir(target)
        files = dir(strcat(target, '\*Day*'));
        latest = files([files(:).datenum] == max([files(:).datenum]));
        fid = fopen(strcat(target, '\', latest.name), 'r');
    else
        fid = fopen(target, 'r');
    end
    try
        site = fgetl(fid);

        for i=1:9 % skip 9 lines
           fgetl(fid);
        end
        type = str2num(fgetl(fid));
        for i=1:3
            fgetl(fid); % skip 3 lines
        end
        line = fgetl(fid);
        nc = length(line);
        comptime = datenum(line(nc-19:nc));

        line = fgetl(fid);
        while isempty(line) || ~strcmp(line(end-1:end), 'ds') % skip forward until the ds command is found
            line = fgetl(fid);
        end

        fgetl(fid);
        line = fgetl(fid);
        date = line(end-11:end);
        fgetl(fid);
        line = fgetl(fid);
        time = line(end-8:end);
        insttime = datenum(strcat(date, time));
        offset = comptime - insttime;
        if abs(offset) > 1/288; % 1/288 = 5 minutes in units of days
            out = ['ALERT: difference between instrument time and computer '...
                   'time is greater than five minutes'];
            if ~IM
                return
            else
                button = questdlg([out ', suggesting a possible typo in the '...
                                       'input file''s header.' char(10) ...
                                       'Do you want to continue anyway?'],...
                         'Large computer/instrument time difference alert',...
                         'Yes', 'No', 'No');
                if isempty(button) || strcmp(button, 'No')
                    return
                end
            end
        end

        line = fgetl(fid);
        while isempty(line) || ~strcmp(line(1:2), 'dd') % skip forward until the dd command is found
            line = fgetl(fid);
        end
        fgetl(fid); % skip a line
        if type == 37 % skip 8 more. This used to be 6--did something change?
            for i=1:8 % If so, when? How far back before this breaks?
                fgetl(fid);
            end
            switch site
                case 'HQ'
                    parsestr = '%n %n %n %n %n %s %s';
                otherwise
                    parsestr = '%n %n %n %s %s';
            end
        else
            parsestr = '%n %n %n %n %n %n %n %n %s %s';
        end
        marker = ftell(fid);
        data = textscan(fid, parsestr, 'Delimiter', ',');
        N = size(data,2);
      % Check for parse error
        nBad = check4ParseError(data);
      % Handle parse error
        if nBad % if nBad is non-zero
            if type == 37 % Parse err probably indicates recs incl. pressure
                type = '37p'; % note this for pushData,
                parsestr = '%n %n %n %n %s %s'; % change the parsing string
                fseek(fid, marker, 'bof'); % rewind, and
                data = textscan(fid, parsestr, 'Delimiter', ','); % rescan the data
                N = size(data,2);
              % check again
                nBad = check4ParseError(data);
            end
            while nBad
              % At this point nBad ~= 0 indicates a different problem,
              % probably incomplete record(s) in the input file.  If that 
              % is the case, the first N-nBad columns (N = total # of cols)
              % will be one longer than last nBad columns and, most 
              % importantly, the file pointer will be situated precisely
              % following the error, i.e., somewhere in the middle of a
              % line.  Thus we can continue by: removing from data any suc-
              % cessful conversions from the beginning of the line and 
              % throwing away the rest of the problem line; calling text-
              % scan again; vertically concatenating the result to data; 
              % checking the result; and repeating this process 'til: nBad 
              % is 0; we encounter a line w/ just "S>"; or we reach end of 
              % file.
              
              % Remove incomplete data line
                for i=1:N-nBad
                    data{i} = data{i}(1:end-1);
                end
              % Get rid of rest of bad line
                fgetl(fid);
              % Call textscan again
                newdata = textscan(fid, parsestr, 'Delimiter', ',');
              % Append newdata to data
                for i=1:N
                    data{i} = [data{i}; newdata{i}];
                end
              % Check new data
                nBad = check4ParseError(newdata);
            end
        end
        fclose(fid);
        out = {offset, site, data, type};
    catch ME
        fclose(fid);
        out = getReport(ME);
    end
end

function nBad = check4ParseError(data)
    check = size(data{1},1);
    nBad = sum(cellfun(@(x) size(x,1)~=check, data(2:end)));
end

function success = pushData(offset_site_data_type, DB, AR)
    global IM
    if isempty(offset_site_data_type) || ischar(offset_site_data_type)
        success = offset_site_data_type;
    else
        if AR, source = -9999;
        else source = -9997; end
        try
            [offset, site, data, type] = deal(offset_site_data_type{:});
            if ischar(type) % 37 w/ pressure
                [T, C, P, S, d, t] = deal(data{:});
                extdata = cell(length(data{1}), 11);
            elseif type == 16 
                [T, C, P, V, A1, A2, A3, S, d, t] = deal(data{:});
                extdata = cell(length(data{1}), 15);
            else % 37 w/out pressure
                [T, C, S, d, t] = deal(data{:});
                extdata = cell(length(data{1}), 10);
            end
            datetimes = zeros(size(t));
            for i=1:length(datetimes)
                extdata(i,1:2) = {0, site};
                if ~(isempty(d{i}) || isempty(t{i}))
                    datetimes(i) = combineDateAndTime(d{i}, t{i});
                end
                if ischar(type)
                    extdata(i,3:8) = {T(i), C(i), P(i), S(i), d{i}, t{i}};
                    extdata{i,9} = datestr(datetimes(i) + offset, 'mm/dd/yyyy HH:MM:SS');
                    extdata{i,10} = source;
                    try
                        extdata{i,11} = datetimes(i)-datetimes(i-1);
                    catch
                        extdata{i,11} = 0;
                    end
                elseif type == 16
                    extdata(i,3:12) = {T(i), C(i), P(i), V(i), A1(i), A2(i), A3(i), S(i), d{i}, t{i}};
                    extdata{i,13} = datestr(datetimes(i) + offset, 'mm/dd/yyyy HH:MM:SS');
                    extdata{i,14} = source;
                    try
                        extdata{i,15} = datetimes(i)-datetimes(i-1);
                    catch
                        extdata{i,15} = 0;
                    end
                else % other surface
                    extdata(i,3:7) = {T(i), C(i), S(i), d{i}, t{i}};
                    extdata{i,8} = datestr(datetimes(i) + offset, 'mm/dd/yyyy HH:MM:SS');
                    extdata{i,9} = source;
                    try
                        extdata{i,10} = datetimes(i)-datetimes(i-1);
                    catch
                        extdata{i,10} = 0;
                    end
                end
            end
            
            start = datetimes(1) - 0.01; % approx. 15 min. earlier than the earliest time
            finish = datetimes(end) + 0.01; % approx. 15 min. later than the last time
            conn = database(DB, '', '');
          % First, get existing data for calculating first SampleInterval
          % and removing duplicates
            sqlQuery = {'SELECT PostedUTCDatetime, ReportedDateDescription, ReportedTimeDescription ',...
                        'FROM RawValue',...
                       ['WHERE SiteID=''' site ''' AND PostedUTCDatetime>=''' datestr(start, 'mm/dd/yyyy HH:MM:SS') ''''...
                        ' AND PostedUTCDatetime<=''' datestr(finish, 'mm/dd/yyyy HH:MM:SS') '''']};
            dbmeta = get(dmd(conn));
            disp(['User ' dbmeta.UserName ' querying SiteID ' site ' in RawValue Table to ' ...
                'eliminate duplicates prior to pushing.']);
            curs = exec(conn, [sqlQuery{1} ' ' sqlQuery{2} ' ' sqlQuery{3}]);
            setdbprefs('DataReturnFormat', 'cellarray')
            curs = fetch(curs);
            close(curs);
            if ~strcmp(curs.Data{1}, 'No Data')
              % Get existing PostedUTCDatetime just preceeding the earliest new PostedUTCDatetime
                PUTCDT0 = datenum(globals_dev.trimPUTCD(curs.Data(:,1)));
                PUTCDT0 = PUTCDT0(PUTCDT0 < datetimes(1));
                PUTCDT0 = max(PUTCDT0);
              % Calculate the first SampleInterval
                extdata{1,end} = datetimes(1)-PUTCDT0;
              % Eliminate duplicates
                datetimes2elim = combineDateAndTime(curs.Data(:,2),...
                                                    curs.Data(:,3));
                indices2elim = arrayfun(@(x) min(abs(datetimes-x)) < 1/288,... % 1/288 = 5 min. in days
                                        datetimes2elim);
                extdata(indices2elim,:) = [];
            end
          % Now, for what remains:
          % Determine and assign QAQC Flags
%            qaqc = QAQC_dev(extdata);
            
          % And insert into the DB
            if ischar(type)
                fastinsert(conn, 'RawValue', ...
                    {'ProcessedFlag', 'SiteID', 'TemperatureQuantity', 'ConductivityQuantity', ...
                     'PressureQuantity', 'SalinityQuantity', 'ReportedDateDescription', ...
                     'ReportedTimeDescription', 'PostedUTCDatetime', 'RealTimeFlagsID', ...
                     'SampleInterval'}, extdata);
            elseif type == 16
                fastinsert(conn, 'RawValue', ...
                    {'ProcessedFlag', 'SiteID', 'TemperatureQuantity', 'ConductivityQuantity', ...
                     'PressureQuantity', 'Aux0Quantity', 'Aux1Quantity', 'Aux2Quantity', ...
                     'Aux3Quantity', 'SalinityQuantity', 'ReportedDateDescription', ...
                     'ReportedTimeDescription', 'PostedUTCDatetime', 'RealTimeFlagsID',...
                     'SampleInterval'}, extdata);
            else % other surface
                fastinsert(conn, 'RawValue', ...
                    {'ProcessedFlag', 'SiteID', 'TemperatureQuantity', 'ConductivityQuantity', ...
                     'SalinityQuantity', 'ReportedDateDescription', 'ReportedTimeDescription', ...
                     'PostedUTCDatetime', 'RealTimeFlagsID', 'SampleInterval'}, extdata);
            end
            close(conn);
            success = 'Success!';
        catch ME
            if exist('conn', 'var')
                close(conn);
            end
            success = getReport(ME);
        end        
    end
end

function reportQueryError(curs, site, report)
    global oEN
    
    if strcmp(curs.Data, 'No Data')
        msg = curs.Data{1};
    else
        msg = curs.Message;
    end
    report = [report sprintf('\n\nHere''s the error message returned by the query:\n\n%s', msg)];
    oEN.reportError('PushData_dev ', site, report, '');
end